# 强化学习 #

import pandas as pd
from db.db import db


class RL(object):
    def __init__(self, reward_decay, learning_rate):
        self.reward_decay = reward_decay  # 奖励衰减值
        self.learning_rate = learning_rate  # 学习率，0-1之间

    def set_states(self, sql):
        df_states = pd.read_sql_query(sql, db.engine, index_col='gid')
        self.df_states = df_states

    def learning(self, sequence):
        """
        输入：马尔科夫反馈过程状态序列
        功能：强化学习，更新状态价值
        """
        for j in range(len(sequence)):
            # 当前状态
            df_state_current = df_states.query(
                f'gid == {sequence.loc[j,"states_id_start"]}')
            # 预测值
            q_predict = df_state_current.loc[df_state_current.index, 'value']
            q_target = 0  # 目标值
            if j != len(sequence) - 1:  # 不是最后一个订单
                df_state_next = df_states.query(
                    f'gid == {sequence.loc[j+1,"states_id_start"]}')  # 下一个状态
                q_target = sequence.loc[
                    j, 'reward'] + reward_decay * df_state_next.loc[
                        df_state_next.index, 'value']
            else:
                q_target = sequence.loc[j, 'reward']
            delta = abs(learning_rate * (q_target - q_predict))  # 误差
            q_value = q_predict + learning_rate * (q_target - q_predict)  # Q值
            self.df_states.loc[df_state_current.index, 'value'] = q_value
            self.df_states.loc[df_state_current.index, 'delta'] = delta
